/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package co.edu.unal.bioingenium.kbmed.experimentation;

import co.edu.unal.bioingenium.kbmed.config.Configuration;
import co.edu.unal.bioingenium.kbmed.knowledge.dao.KnowledgeSourceDAO;
import co.edu.unal.bioingenium.kbmed.knowledge.dao.KnowledgeSourceDAOFactory;
import co.edu.unal.bioingenium.kbmed.retrieval.knowledge.process.PrepareByConcepts;
import co.edu.unal.bioingenium.kbmed.retrieval.text.process.PrepareByWords;
import co.edu.unal.bioingenium.kbmed.text.index.io.InvertedIndexIO;
import co.edu.unal.bioingenium.kbmed.text.index.structures.InvertedIndexStrToInt;
import co.edu.unal.bioingenium.kbmed.text.representation.io.CorpusIO;
import co.edu.unal.bioingenium.kbmed.text.representation.vo.Corpus;
import co.edu.unal.bioingenium.kbmed.util.io.FileManager;
import co.edu.unal.bioingenium.kbmed.util.time.TimeUtil;
import gnu.trove.map.TObjectIntMap;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author Alejandro Riveros Cruz
 */
public class CollectionStatistics {
    //Words statistics 
    //  Global and by specialty:
    //   Total words 
    //   Mean words by document
    //   Most frequent words
    //   Maximun TFIDF

    //Concept statistics 
    //  Global and by specialty:
    //   Total concepts  
    //   Mean concepts by document
    //   Most frequent concepts
    //   Maximun TFIDF
    //   Total categories
    //   Mean categories by document
    //   Most frequent categories
    //   Words Vs concepts by document, and also grouped by specialty
    private static final String[] saruroSpecialtys = {
        "cardiologia",
        "cuidado intermedio",
        "dermatologia",
        "ginecologia y obstetricia",
        "infectologia",
        "mamografia",
        "medicina interna",
        "neurologia",
        "nutricion humana",
        "ortopedia",
        "otorrinolaringologia",
        "patologia",
        "patologia anatomica y clinica",
        "pediatria",
        "psiquiatria",
        "radiologia",
        "reumatologia",
        "urologia"
    };
    private final String startDate;

    public CollectionStatistics() {
        startDate = TimeUtil.getCurrentDate();
    }

    public static void main(String[] args) throws IOException, ClassNotFoundException {
        CollectionStatistics collectionStatistics = new CollectionStatistics();
        collectionStatistics.wordsStatistics("instrumentos", FileManager.toStandardPath("/media/ldatos/instrumentos"));
        collectionStatistics.conceptStatistics("instrumentos", FileManager.toStandardPath("/media/ldatos/instrumentos"));
        if (args.length < 2) {
            System.out.println("Usage:");
            System.out.println("CollectionStatistics corpusName corpusPath");
            System.exit(1);
        } else {
//            CollectionStatistics collectionStatistics = new CollectionStatistics();

            collectionStatistics.wordsStatistics(args[0], FileManager.toStandardPath(args[1]));
            collectionStatistics.conceptStatistics(args[0], FileManager.toStandardPath(args[1]));
        }

    }

    public void wordsStatistics(String corpusName, String corpusPath) throws IOException, ClassNotFoundException {
        Corpus corpus = CorpusIO.loadCorpus(corpusName + "WordsCorpus.dat");
        //Global
        saveCorpusStatistics(corpusName + "WordsGlobal", corpus, false);
        //By Specialty
        PrepareByWords prepareCollection = new PrepareByWords();
        Corpus tempCorpus;
        InvertedIndexStrToInt documentByMeta = InvertedIndexIO.loadStrToIntInvertedIndex(corpusName + "MetaByDocument.idx");
        List<String> wholeDocumentList = corpus.getSortedDocumentList();
        Set<Integer> documentsIdx;
        List<String> currentDocuemntList;
        String specialtyFixedName;
        for (int i = 0; i < saruroSpecialtys.length; i++) {
            specialtyFixedName = saruroSpecialtys[i].toUpperCase().replace(" ", "_");
            System.out.println(specialtyFixedName);
            documentsIdx = documentByMeta.getElement(saruroSpecialtys[i]);
            currentDocuemntList = new ArrayList<String>();
            for (Integer documentIdx : documentsIdx) {
                currentDocuemntList.add(wholeDocumentList.get(documentIdx));
            }
            tempCorpus = prepareCollection.buildCorpus(corpusName + saruroSpecialtys[i] + "Words", corpusPath, currentDocuemntList, 0, 0);
            CorpusIO.storeCorpus(tempCorpus, corpusName + specialtyFixedName + "WordsCorpus.dat");
            saveCorpusStatistics(corpusName + specialtyFixedName + "Words", tempCorpus, false);
        }
    }

    public void conceptStatistics(String corpusName, String corpusPath) throws IOException, ClassNotFoundException {
        Corpus corpus = CorpusIO.loadCorpus(corpusName + "ConceptsCorpus.dat");
        //Global
        saveCorpusStatistics(corpusName + "ConceptsGlobal", corpus, true);
        //By Specialty
        PrepareByConcepts prepareCollection = new PrepareByConcepts();
        prepareCollection.initKnowledgeSource();
        Corpus tempCorpus;
        InvertedIndexStrToInt documentByMeta = InvertedIndexIO.loadStrToIntInvertedIndex(corpusName + "MetaByDocument.idx");
        List<String> wholeDocumentList = corpus.getSortedDocumentList();
        Set<Integer> documentsIdx;
        List<String> currentDocuemntList;
        String specialtyFixedName;
        for (int i = 0; i < saruroSpecialtys.length; i++) {
            specialtyFixedName = saruroSpecialtys[i].toUpperCase().replace(" ", "_");
            System.out.println(specialtyFixedName);
            documentsIdx = documentByMeta.getElement(saruroSpecialtys[i]);
            currentDocuemntList = new ArrayList<String>();
            for (Integer documentIdx : documentsIdx) {
                currentDocuemntList.add(wholeDocumentList.get(documentIdx));
            }
            tempCorpus = prepareCollection.buildConceptCorpus(corpusName + saruroSpecialtys[i] + "Concepts", corpusPath, currentDocuemntList, 0, 0);
            CorpusIO.storeCorpus(tempCorpus, corpusName + specialtyFixedName + "ConceptCorpus.dat");
            saveCorpusStatistics(corpusName + specialtyFixedName + "Concepts", tempCorpus, true);
        }
    }

    public void saveCorpusStatistics(String name, Corpus corpus, boolean isConcept) {
        KnowledgeSourceDAO knowledgeSourceDAO = KnowledgeSourceDAOFactory.getKnowledgeSourceDAO();
        BufferedWriter bufferedWriter = null;
        try {
            String savePath = Configuration.EXPERIMENTS_DIR + File.separator + CollectionStatistics.class.getSimpleName() + File.separator + startDate;
            FileManager.verifyAndCreatePath(savePath);
            bufferedWriter = new BufferedWriter(new FileWriter(FileManager.toStandardPath(savePath) + name + "Statistics.csv"));
            bufferedWriter.write("Total_documents " + corpus.getTotalDocuments() + "\n");
            bufferedWriter.write("Average_document_length " + corpus.getAverageDocumentLength() + "\n");
            bufferedWriter.write("Total_words " + corpus.getTotalWords() + "\n");
            TObjectIntMap<String> wordList = corpus.getWordCount();
            if (isConcept) {
                bufferedWriter.write("word,term,totalOcurrences,inDocumentOccurrences\n");
            } else {
                bufferedWriter.write("word,totalOcurrences,inDocumentOccurrences\n");
            }
            StringBuilder sb = new StringBuilder();
            for (String word : wordList.keySet()) {
                sb.setLength(0);
                if (isConcept) {
                    sb.append(word).append(",").append(knowledgeSourceDAO.getTermForDescriptorId(word)).append(",").append(wordList.get(word)).append(",").append(corpus.getDocumentOcurrences(word)).append("\n");
                } else {
                    sb.append(word).append(",").append(wordList.get(word)).append(",").append(corpus.getDocumentOcurrences(word)).append("\n");
                }
                bufferedWriter.write(sb.toString());
            }
            bufferedWriter.flush();
            bufferedWriter.close();

        } catch (IOException ex) {
            Logger.getLogger(CollectionStatistics.class.getName()).log(Level.SEVERE, null, ex);
        } finally {
            try {
                bufferedWriter.close();
            } catch (IOException ex) {
                Logger.getLogger(CollectionStatistics.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
    }
}
